temp =  read.spss(file = files[i]) %>% data.frame(., stringsAsFactors = F)
files = c(list.files(pattern = ".sav"), list.files(pattern = ".POR"))
n = length(files)
for(i in 1:n){
temp =  read.spss(file = files[i]) %>% data.frame(.)
filename = gsub("[^0-9\\]", "", files[i])  %>% paste("GEM", ., sep = "_")
assign(filename, temp)
}
install.packages("haven")
library(haven)
dir = here()
setwd(dir)
# read in data
files = c(list.files(pattern = ".sav"), list.files(pattern = ".POR"))
n = length(files)
i = 10
temp =  read_spss(file = files[i]) %>% data.frame(.)
library(microbenchmark)
microbenchmark(
temp =  read_spss(file = files[i]) ,
temp.b =  read.spss(file = files[i]) ,
times = 1
)
View(temp)
temp =  read_spss(file = files[i]) %>% data.table(.)
View(temp)
library(haven)
library(data.table)
library(here)
library(magrittr)
dir = here()
setwd(dir)
# read in data
files = c(list.files(pattern = ".sav"), list.files(pattern = ".POR"))
n = length(files)
for(i in 1:n){
temp =  read_spss(file = files[i]) %>% data.table(.)
filename = gsub("[^0-9\\]", "", files[i])  %>% paste("GEM", ., sep = "_")
assign(filename, temp)
}
library(haven)
library(data.table)
library(here)
library(magrittr)
dir = here()
setwd(dir)
# read in data
files = c(list.files(pattern = ".sav"), list.files(pattern = ".POR"))
n = length(files)
i = 1
library(haven)
library(data.table)
library(here)
library(magrittr)
dir = here()
setwd(dir)
# read in data
files = c(list.files(pattern = ".sav"), list.files(pattern = ".POR"))
n = length(files)
i = 1
GEM =  read_spss(file = files[i]) %>% data.table(.)
variables = colnames(GEM)
variables = colnames(GEM)
grep("omnowjob", variables)
?grep
grep("omnowjob", variables, ignore.case = T)
firm_size = grep("omnowjob", variables, ignore.case = T) %>% GEM[,.]
firm_size = grep("omnowjob", variables, ignore.case = T)
location = grep("omnowjob", variables, ignore.case = T)
GEM[, location]
test = GEM[, 2]
GEM =  read_spss(file = files[i]) %>% data.frame(.)
variables = colnames(GEM)
location = grep("omnowjob", variables, ignore.case = T)
test = GEM[, location]
View(GEM)
test
test = GEM[, location] %>% as.vector()
test
test = GEM[, location] %>% as.character()
test = GEM[, location] %>% as.numeric()
location = grep("country", variables, ignore.case = T)
location = grep("country", variables, ignore.case = T)
country = GEM[, location] %>% as.character()
country = GEM[, location]
View(GEM)
location = grep("country", variables, ignore.case = T)
country = GEM[, location] %>% as.numeric()
location = grep("year", variables, ignore.case = T)
year = gsub("[^0-9]", "", file[i])
year = gsub("[^0-9]", "", files[i])
result = data.table(country, year, firm_size)
View(result)
location = grep("omnowjob", variables, ignore.case = T)
firm_size = GEM[, location] %>% as.numeric()
location = grep("country", variables, ignore.case = T)
country = GEM[, location] %>% as.numeric()
# variable list
variables = colnames(GEM)
# get firm size
location = grep("omnowjob", variables, ignore.case = T)
firm_size = GEM[, location] %>% as.numeric()
# country
location = grep("country", variables, ignore.case = T)
country = GEM[, location] %>% as.numeric()
year = gsub("[^0-9]", "", files[i])
result = data.table(country, year, firm_size)
View(result)
hist(result$firm_size)
max(firm_size)
max(firm_size, na.rm = T)
GEM =  read_spss(file = files[i]) %>% data.frame(.)
# variable list
variables = colnames(GEM)
# get firm size
location = grep("omnowjob", variables, ignore.case = T)
firm_size = GEM[, location] %>% as.character()
# country
location = grep("country", variables, ignore.case = T)
country = GEM[, location] %>% as.numeric()
year = gsub("[^0-9]", "", files[i])
result = data.table(country, year, firm_size)
View(result)
library(haven)
library(data.table)
library(here)
library(magrittr)
dir = here()
setwd(dir)
# read in data
files = c(list.files(pattern = ".sav"), list.files(pattern = ".POR"))
n = length(files)
final = NULL
for(i in 1:n){
GEM =  read_spss(file = files[i]) %>% data.frame(.)
# variable list
variables = colnames(GEM)
# get firm size
location = grep("omnowjob", variables, ignore.case = T)
firm_size = GEM[, location] %>% as.numeric()
# country
location = grep("country", variables, ignore.case = T)
country = GEM[, location] %>% as.numeric()
year = gsub("[^0-9]", "", files[i])
result = data.table(country, year, firm_size)
final = rbind(final, result)
}
View(final)
hist(final$country)
readme = files[i]
grepl("POR", readme)
readme = files[i]
if( grepl("sav", readme) ){
GEM =  read_spss(file = ) %>% data.frame(.)
} else {
GEM = read.spss(file = ) %>% data.frame(.)
}
readme = files[i]
if( grepl("sav", readme) ){
GEM =  read_spss(file = readme ) %>% data.frame(.)
} else {
GEM = read.spss(file = readme) %>% data.frame(.)
}
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/GEM/Raw Data/Format.R')
mean(final
$firm_size)
max(final$firm_size, na.rm = T)
fwrite(final, "GEM_merge.csv")
library(haven)
library(data.table)
library(here)
library(magrittr)
dir = here()
setwd(dir)
GEM = gread("GEM_raw.csv")
GEM = fread("GEM_raw.csv")
GEM = fread("GEM_raw.csv") %>% na.omit()
GEM = GEM[! GEM$firm_size %in%  omit]
omit = c(999999, 999998)
GEM = GEM[! GEM$firm_size %in%  omit]
min(GEM$firm_size)
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/GEM/Raw Data/Clean.R')
View(GEM)
country_codes = fread("country codes.csv"")
country_codes = fread("country codes.csv")
country_codes = fread("country codes.csv")
View(country_codes)
?merge
test = merge(GEM, country_codes)
names(GEM$country)
View(country_codes)
names(GEM) = c("country_numerical", "year", "firm_size" )
names(country_codes) = c("country", "code2", "code3", "country_numerical")
test = merge(GEM, country_codes, by = country_numerical)
test = merge(GEM, country_codes, by = "country_numerical")
View(test)
unique(GEM$country_numerical)
unique(GEM$country_numerical) %>% sort()
unique(country_codes$country_numerical)
unique(country_codes$country_numerical) %>% sort()
intersect(GEM$country_numerical, country_codes$country_numerical)
intersect(GEM$country_numerical, country_codes$country_numerical) %>% length()
i = 10
library(haven)
library(foreign)
library(data.table)
library(here)
library(magrittr)
dir = here()
setwd(dir)
# read in data
files = c(list.files(pattern = ".sav"), list.files(pattern = ".POR"))
n = length(files)
final = NULL
readme = files[i]
GEM =  read_spss(file = readme )
View(GEM)
GEM$country
readme = files[i]
if( grepl("sav", readme) ){
GEM =  read_spss(file = readme )
} else {
GEM = read.spss(file = readme)
}
variables = colnames(GEM)
location = grep("omnowjob", variables, ignore.case = T)
firm_size = GEM[, location] %>% as.numeric()
location = grep("country", variables, ignore.case = T)
country = GEM[, location] %>% as.numeric()
country = GEM[, location]
country$country
labels(country)
x = labels(country)
country[[1]]
country[[2]]
attributes(country)
GEM[, location]
country = GEM[, location]
country
country[[3]]
country[[1]]
View(country)
as.factor(country, "labels")
print_labels(country)
print_labels(GEM)
?haven::labelled
is.labelled(country)
test = as_factor(country)
View(test)
test = as_factor(country) %>% as.character()
test = as_factor(country)
data.table(test)
View(test)
country = GEM[, location] %>% as_factor()
View(country)
location = grep("omnowjob", variables, ignore.case = T)
firm_size = GEM[, location] %>% as.numeric()
location = grep("omnowjob", variables, ignore.case = T)
firm_size = GEM[, location]
firm_size = GEM[, location] %>% as.vector()
View(firm_size)
readme = files[i]
if( grepl("sav", readme) ){
GEM =  read_spss(file = readme )
} else {
GEM = read.spss(file = readme)
}
# variable list
variables = colnames(GEM)
# country
location = grep("country", variables, ignore.case = T)
country = GEM[, location] %>% as_factor()
# get firm size
location = grep("omnowjob", variables, ignore.case = T)
firm_size = GEM[, location] %>% as.vector()
year = gsub("[^0-9]", "", files[i])
result = data.table(country, year, firm_size)
final = rbind(final, result)
View(result)
fwrite(final, "GEM_raw.csv")
result = data.table(country, year, firm_size = firm_size)
View(result)
firm_size = GEM[, location] %>% data.table()
View(firm_size)
result = data.table(country, year, firm_size = firm_size)
View(result)
firm_size = GEM[, location] %>% as.numeric()
# country
location = grep("country", variables, ignore.case = T)
country = GEM[, location] %>% as_factor()
# get firm size
location = grep("omnowjob", variables, ignore.case = T)
firm_size = GEM[, location]
year = gsub("[^0-9]", "", files[i])
result = data.table(country, year, firm_size = firm_size)
final = rbind(final, result)
firm_size = GEM[, location] %>% as.vector()
year = gsub("[^0-9]", "", files[i])
result = data.table(country, year, firm_size = firm_size)
final = rbind(final, result)
final = NULL
# variable list
variables = colnames(GEM)
# country
location = grep("country", variables, ignore.case = T)
country = GEM[, location] %>% as_factor()
# get firm size
location = grep("omnowjob", variables, ignore.case = T)
firm_size = GEM[, location] %>% as.vector()
year = gsub("[^0-9]", "", files[i])
result = data.table(country, year, firm_size = firm_size)
final = rbind(final, result)
# variable list
variables = colnames(GEM)
# country
location = grep("country", variables, ignore.case = T)
country = GEM[, location] %>% as_factor()
# get firm size
location = grep("omnowjob", variables, ignore.case = T)
firm_size = GEM[, location] %>% as.vector()
year = gsub("[^0-9]", "", files[i])
result = data.table(country, year, firm_size = firm_size)
final = rbind(final, result)
View(final)
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/GEM/Raw Data/Format.R')
print(i)
variables = colnames(GEM)
location = grep("country", variables, ignore.case = T)
country = GEM[, location] %>% as_factor()
files = list.files(pattern = ".sav")
n = length(files)
final = NULL
files
print(i)
print(i, "of", n)
print(i)
print(n)
print(i)     print(n)
print(i, n)
print(i/n)
print(i/n*100)
i=1
print(i/n*100)
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/GEM/Raw Data/Format.R')
library(haven)
library(data.table)
library(here)
library(magrittr)
dir = here()
setwd(dir)
GEM = fread("GEM_raw.csv") %>% na.omit()
names(GEM) = c("country_numerical", "year", "firm_size" )
country_codes = fread("country codes.csv")
names(country_codes) = c("country", "code2", "code3", "country_numerical")
intersect(GEM, country_codes$country)
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/GEM/Raw Data/clean.R')
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/GEM/Raw Data/clean.R')
View(codes)
names(GEM) = c("country", "year", "firm_size" )
codes = fread("country codes.csv")
grep(GEM$country, codes$Country[1])
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/GEM/Raw Data/clean.R', echo=TRUE)
grep(codes$Country[1], GEM$country)
GEM = fread("GEM_raw.csv") %>% na.omit()
# remove non-response codes 99999, 999998
omit = c(999999, 999998)
GEM = GEM[! GEM$firm_size %in%  omit]
names(GEM) = c("country", "year", "firm_size" )
codes = fread("country codes.csv")
GEM = fread("GEM_raw.csv") %>% na.omit()
names(GEM) = c("country", "year", "firm_size" )
omit = c(999999, 999998)
GEM = GEM[! GEM$firm_size %in%  omit]
GEM = fread("GEM_raw.csv") %>% na.omit()
names(GEM) = c("country", "year", "firm_size" )
# remove non-response codes 99999, 999998
omit = c(999999, 999998)
GEM = GEM[! GEM$firm_size %in%  omit]
codes = fread("country codes.csv")
grep(codes$Country[1], GEM$country)
View(GEM)
View(GEM)
grep(codes$Country[2], GEM$country)
View(codes)
View(GEM)
View(codes)
grep(codes$Country[87], GEM$country)
View(codes)
codes$trunc = strtrim(codes$Country, 5)
View(codes)
codes$trunc = strtrim(codes$Country, 6)
View(codes)
View(codes)
test = unique(GEM$country)
test
test = unique(GEM$country) %>% sort()
test
View(codes)
fwrite(GEM.countries, "GEM_names.csv")
GEM.countries = unique(GEM$country) %>% sort()
fwrite(GEM.countries, "GEM_names.csv")
GEM.countries = unique(GEM$country) %>% sort() %>% data.table()
fwrite(GEM.countries, "GEM_names.csv")
intersect(codes$Country, unique(GEM$country))
matching_names = intersect(codes$Country, unique(GEM$country))
GEM_countries = unique(GEM$country)
GEM_countries = unique(GEM$country)
matching_names = intersect(codes$Country, GEM_countries)
non_matching = GEM_countries[! GEM_countries %in% matching_names]
non_matching = GEM_countries[! GEM_countries %in% matching_names]
non_matching
View(codes)
View(codes)
GEM$country[ GEM$country == "Kazakstan"] = "Kazakhstan"
#mispellings
GEM$country[ GEM$country == "Kazakstan"] = "Kazakhstan"
GEM_countries = unique(GEM$country)
matching_names = intersect(codes$Country, GEM_countries)
non_matching = GEM_countries[! GEM_countries %in% matching_names]
non_matching
View(codes)
i=1
grep(non_matching[i], codes$Country)
grep(non_matching[i], codes$Country)
find = grep(non_matching[i], codes$Country)
codes$Country[find]
rename =  codes$Country[find]
GEM$country[GEM$country == non_matching[i]] = rename
n = length(non_matching)
i = 1
gem_name = non_matching[i]
find = grep(gem_name, codes$Country)
gem_name = non_matching[i]
find = grep(gem_name, codes$Country)
rename =  codes$Country[find]
GEM$country[GEM$country == non_matching[i]] = rename
for(i in 1:n){
gem_name = non_matching[i]
# find location in correct codes and rename
find = grep(gem_name, codes$Country)
rename =  codes$Country[find]
GEM$country[GEM$country == non_matching[i]] = rename
}
gem_name = non_matching[i]
find = grep(gem_name, codes$Country)
find = grep(gem_name, codes$Country)
rename =  codes$Country[find]
length(rename)
if( length(rename) == 0){
}
else{
GEM$country[GEM$country == non_matching[i]] = rename
}
if( length(rename) == 0){}
else{
GEM$country[GEM$country == non_matching[i]] = rename
}
else{       GEM$country[GEM$country == non_matching[i]] = rename     }
if( length(rename) == 0){}
else{       GEM$country[GEM$country == non_matching[i]] = rename     }
if( length(rename) == 0){}     else{       GEM$country[GEM$country == non_matching[i]] = rename     }
if( length(rename) == 0){}     else{       GEM$country[GEM$country == non_matching[i]] = rename     }
length(rename) == 0
?if
GEM$country[GEM$country == non_matching[i]] = rename
if( length(rename) == 0){
x = 1
}    else{       GEM$country[GEM$country == non_matching[i]] = rename     }
if( length(rename) == 0){
x = 1
}    else{       GEM$country[GEM$country == non_matching[i]] = rename     }
if( length(rename) == 0){
x = 1
}    else{       GEM$country[GEM$country == non_matching[i]] = rename     }
gem_name = non_matching[i]
# find location in correct codes and rename
find = grep(gem_name, codes$Country)
rename =  codes$Country[find]
if( length(rename) == 1){
GEM$country[GEM$country == non_matching[i]] = rename
}
for(i in 1:n){
gem_name = non_matching[i]
# find location in correct codes and rename
find = grep(gem_name, codes$Country)
rename =  codes$Country[find]
if( length(rename) == 1){
GEM$country[GEM$country == non_matching[i]] = rename
}
}
non_matching
View(GEM)
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/GEM/Raw Data/clean.R')
GEM = GEM[country %in% codes$Country]
names(codes$Country) = "country"
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/GEM/Raw Data/clean.R')
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/GEM/Raw Data/clean.R')
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/GEM/Raw Data/clean.R')
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/GEM/Raw Data/clean.R')
test = merge(GEM, codes, by = "country")
View(test)
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/GEM/Raw Data/clean.R')
GEM = GEM[country %in% codes$country]
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/GEM/Raw Data/clean.R')
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/GEM/Raw Data/clean.R')
View(final)
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/GEM/Raw Data/clean.R')
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/GEM/Raw Data/clean.R')
View(GEM_clean)
test = GEM[, mean(firm_size), by = country]
View(test)
source('~/Desktop/origin_inequality/Supplementary Material/Empirical Data/GEM/Raw Data/clean.R')
